package com.chance.cc.crawler.development.bootstrap.toutiao;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.downloader.*;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import groovy.util.logging.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.xml.ws.spi.http.HttpContext;
import java.io.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
public class TouTiaoSignatureTest {
    public static Logger logger = LoggerFactory.getLogger(TouTiaoSignatureTest.class);
    public static final String flushListUrlFormat = "https://www.toutiao.com/api/pc/list/feed?channel_id=%s&max_behot_time=%s&category=pc_profile_channel";
    public static final String flushListUrlFormatStart = "https://www.toutiao.com/api/pc/list/feed?channel_id=%s&min_behot_time=0&refresh_count=1&category=pc_profile_channel";
    public static LinkedBlockingQueue<String> uaList= new LinkedBlockingQueue<>();


    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {

        HttpConfig httpConfig = HttpConfig.me("get");
        httpConfig.setResponseTextGenerateHtml(false);
//        httpConfig.setCircularRedirectsAllowed(false);
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i=1;i<=1;i++){
            new Thread(new TestRun(downloader,httpConfig),"thread_"+i).start();
        }
    }

    static class TestRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String path = "E:\\source\\ua.txt";
            getUaList(path);
            Boolean cire = true;
            String url = "https://www.toutiao.com/api/pc/list/feed?channel_id=3189399007&min_behot_time=0&refresh_count=1&category=pc_profile_channel";
            HttpRequest httpRequest = new HttpRequest();
            Map<String, String> signature = getSignature(url);
            httpRequest.setUrl(url + "&_signature=" + signature.get("signature"));
            httpRequest.setMethod("GET");
            httpRequest.addHeader("User-Agent",signature.get("ua"));
            HttpPage httpPage = downloader.download(httpRequest, httpConfig);
            int i = 1;
            Map<String,Object> map = new HashMap<>();
            map.put("num",i);
            httpRequest.setExtras(map);
            while (cire){
                if(StringUtils.isBlank(httpPage.getRawText())){
                    cire=false;
                }
                i = i + 1;
                if(i > 1){
                    httpRequest = getNextPage(httpPage);
                }
                httpPage = downloader.download(httpRequest, httpConfig);
                Map<String,Object> map1 = new HashMap<>();
                map.put("num",i);
                httpPage.getRequest().setExtras(map1);
            }
            System.out.println(i);
            System.out.println(httpPage.getRawText());
            System.out.println(JSONObject.toJSONString(httpRequest));
        }


        private Map<String,String> getSignature(String url){
            String requestUrl = "http://192.168.1.8:8899/encrypt/toutiao/signature";
            Map<String,Object> body = new HashMap<>();
            String oneUa = getOneUa();
            if(StringUtils.isBlank(oneUa)){
                getSignature(url);
            }
            body.put("url",url);
            body.put("ua",oneUa);
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(requestUrl);
            httpRequest.setMethod(HttpConstant.Method.POST);
            httpRequest.setRequestBody(HttpRequestBody.form(body,"UTF-8"));
            HttpConfig httpConfig = HttpConfig.me("getSignature");
            HttpPage download = downloader.download(httpRequest, httpConfig);
            Map<String,String> result = new HashMap<>();
            if(download != null){
                result.put("signature", download.getRawText());
                result.put("ua",oneUa);
                return result;
            }else{
                getSignature(url);
            }
            return result;
        }

        private HttpRequest getNextPage(HttpPage httpPage){
            String httpRequestUrl = httpPage.getRequest().getUrl();
            Map<String, Object> urlParams = getUrlParams(httpRequestUrl);
            JSONObject pageObj = JSONObject.parseObject(httpPage.getRawText());
            boolean hasMore = pageObj.getBooleanValue("has_more");
            JSONArray dataNodes = pageObj.getJSONArray("data");
            if(dataNodes == null || dataNodes.size() < 1){
                HttpRequest request = httpPage.getRequest();
                httpPage = downLoad(request);
                dataNodes = JSONObject.parseObject(httpPage.getRawText()).getJSONArray("data");
            }

            if (null != urlParams && hasMore){
                String channel_id = (String) urlParams.get("channel_id");
                String maxBeHotTime = dataNodes.getJSONObject(dataNodes.size() - 1).getString("behot_time");
                String nextUrl = String.format(flushListUrlFormat,channel_id,maxBeHotTime);
                HttpRequest httpRequest = new HttpRequest();
                Map<String, String> signature = getSignature(nextUrl);
                httpRequest.setUrl(nextUrl + "&_signature="+signature.get("signature"));
                httpRequest.setMethod("GET");
                httpRequest.addHeader("User-Agent",signature.get("ua"));
                httpRequest.setExtras(httpPage.getRequest().getExtras());
                return httpRequest;
            }
            return null;
        }

        private HttpPage downLoad(HttpRequest httpRequest){
            HttpPage download = downloader.download(httpRequest, httpConfig);
            if(download == null){
                downLoad(httpRequest);
            }
            return download;
        }

        private Map<String, Object> getUrlParams(String url) {
            Map<String, Object> map = new HashMap<String, Object>(0);
            String param = url;
            if (url.contains("?")) {
                param = url.split("\\?")[1];
            }
            if (StringUtils.isBlank(param)) {
                return null;
            }
            String[] params = param.split("&");
            for (String s : params) {
                String[] p = s.split("=");
                if (p.length == 2) {
                    map.put(p[0], p[1]);
                }
            }
            return map;
        }

        private void getUaList(String path){
            try {
                BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path),"UTF-8"));
                String line = null;
                while ((line = bufferedReader.readLine()) != null){
                    if(StringUtils.isBlank(line)){
                        continue;
                    }
                    uaList.add(line);
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        private String getOneUa(){
            try {
                String ua = uaList.poll(3, TimeUnit.MILLISECONDS);
                uaList.add(ua);
                return ua;
            }catch (Exception e){
                e.printStackTrace();
            }
            return null;
        }

    }


}
